package com.baidu.mpks.statistic.service;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baidu.mpks.dal.dataaccessplatform.DataAccessSourceDao;
import com.baidu.mpks.dal.enumeration.EnumDao;
import com.baidu.mpks.statistic.cache.CacheService;
import com.baidu.mpks.statistic.cache.LocalCache;
import com.baidu.mpks.statistic.elasticSearch.EsService;
import com.baidu.mpks.statistic.meta.Action;
import com.baidu.mpks.statistic.meta.ClickRatioCount;
import com.baidu.mpks.statistic.meta.DateCount;
import com.baidu.mpks.statistic.meta.QueryCount;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.baidu.mpks.backendweb.source.service.SourceService;
import com.baidu.mpks.backendweb.ktree.service.TreeService;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.range.Range;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.Cardinality;
import org.redisson.client.RedisClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;

import javax.annotation.Resource;
import javax.validation.constraints.NotNull;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;


@Slf4j
@Service
public class StatisticsService {
    @Resource
    EsService esService;
    @Resource
    CacheService cacheService;
    @Autowired
    RedisClient redisClient;

    @Autowired
    SourceService sourceService;
    @Autowired
    TreeService treeService;
    @Autowired
    EnumDao enumDao;

    @Autowired
    DataAccessSourceDao dataAccessSourceDao;

    @Value("${shenfei.ldapUrl}")
    public String ldapUrl;
    @Value("${shenfei.nameSpace}")
    public String nameSpace;
    @Value("${shenfei.personMethod}")
    public String personMethod;
    @Value("${shenfei.deptMethod}")
    public String deptMethod;
    @Value("${shenfei.timeliness.check}")
    public String needCheckSystem;

    private static final Logger LOGGER = LoggerFactory.getLogger(StatisticsService.class);
    private static ObjectMapper objectMapper = new ObjectMapper();


    public long getPageView(long beginTs, long endTs) throws Exception {
        long pv;
        String cacheKey = "pv_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            pv = (long) cache;
        } else {
            pv = esService.getActionCount(Action.SEARCH, beginTs, endTs);
            cacheService.saveDataToDbAndCache(cacheKey, pv, LocalCache.EXPIRE_UNIT_HOUR);
        }
        LOGGER.debug("pv: " + pv + ", use cache: " + (cache != null));
        return pv;
    }

    public List<DateCount> getPageView(long beginTs, long endTs, String interval) throws Exception {
        List<DateCount> counts;
        String cacheKey = "pv_" + interval + "_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            counts = (List<DateCount>) cache;
        } else {
            counts = esService.getActionHistogram(Action.SEARCH, beginTs, endTs, interval, null);
            cacheService.saveDataToDbAndCache(cacheKey, counts, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("pv per day: " + counts + ", use cache: " + (cache != null));
        return counts;
    }

    public long getUserView(long beginTs, long endTs) throws Exception {
        long uv;
        // 只统计登录成功的,登录失败不应该统计
        // code为0,表示登录成功
        BoolQueryBuilder boolQueryBuilder =
                QueryBuilders.boolQuery().must(QueryBuilders.termQuery(EsService.FIELD_URI, Action.LOGIN.toString()))
                        .must(QueryBuilders.termQuery(EsService.FIELD_CODE, "0"));
        uv = esService.getActionCount(Action.LOGIN, beginTs, endTs, boolQueryBuilder);
        LOGGER.debug("uv: " + uv);
        return uv;
    }

    public List<DateCount> getUserView(long beginTs, long endTs, String interval) throws Exception {
        List<DateCount> counts;
        String cacheKey = "uv_" + interval + "_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            counts = (List<DateCount>) cache;
        } else {
            // counts = esService.getActionHistogram(Action.LOGIN, beginTs, endTs, interval, null);
            // todo: 当前方案统计uv按照搜索用户统计的，并不准确，后续需要涉及用户操作的所有模块加上打点日志
            counts = getSearchUserViewPerDay(beginTs, endTs, interval);
            cacheService.saveDataToDbAndCache(cacheKey, counts, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("uv per day: " + counts + ", use cache: " + (cache != null));
        return counts;
    }

    public List<DateCount> getSearchUserViewPerDay(long beginTs, long endTs, String interval) throws Exception {
        List<DateCount> counts;
        String cacheKey = "suv_" + interval + "_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            counts = (List<DateCount>) cache;
        } else {
            BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
                    .mustNot(QueryBuilders.termQuery(FIELD_USERID, ""));
            counts = esService.getActionHistogram(Action.SEARCH, new String[]{FIELD_USERID}, beginTs, endTs, interval,
                    queryBuilder);
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery()
                    .mustNot(QueryBuilders.existsQuery(FIELD_USERID));
            List<DateCount> countSession = esService.getActionHistogram(Action.SEARCH, new String[]{FIELD_SESSIONID},
                    beginTs, endTs, interval, boolQueryBuilder);
            LOGGER.debug("session count : {}", countSession);
            countSession.forEach(c -> {
                int index = counts.indexOf(c);
                if (index >= 0) {
                    DateCount dateCount = counts.get(index);
                    dateCount.setCount(dateCount.getCount() + c.getCount());
                } else {
                    counts.add(c);
                }
            });
            counts.sort(Comparator.comparing(DateCount::getDate));
            cacheService.saveDataToDbAndCache(cacheKey, counts, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("suv per day: " + counts + ", use cache: " + (cache != null));
        return counts;
    }

    public JsonNode getCounts(long beginTs, long endTs) throws Exception {
        long uv = getUserView(beginTs, endTs);
        long clickCount = getClickCount(beginTs, endTs);
        long agreeCount = getAgreedCount(beginTs, endTs);
        long disagreeCount = getDisagreedCount(beginTs, endTs);
        long turnCount = getTurnQueryCount(beginTs, endTs);
        long hitCount = getHitQueryCount(beginTs, endTs);
        long searchCount = getSearchCount(beginTs, endTs);
        ObjectNode result = objectMapper.createObjectNode();
        result.put("login", uv);
        result.put("search", searchCount);
        result.put("hit", hitCount);
        result.put("click", clickCount);
        result.put("agreed", agreeCount);
        result.put("disagreed", disagreeCount);
        result.put("turn", turnCount);
        return result;
    }

    public Long getAgreedCount(long beginTs, long endTs) throws Exception {
        Long count;
        String cacheKey = "agree_count_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            count = (Long) cache;
        } else {
            count = esService.getDistinctCount(Action.LIKE, FIELD_DATAID, beginTs, endTs, null);
            cacheService.saveDataToDbAndCache(cacheKey, count, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("agreed and agreed: " + count + ", use cache: " + (cache != null));
        return count;
    }

    public Long getDisagreedCount(long beginTs, long endTs) throws Exception {
        Long count;
        String cacheKey = "disagree_count_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            count = (Long) cache;
        } else {
            count = esService.getDistinctCount(Action.DISLIKE, FIELD_DATAID, beginTs, endTs, null);
            cacheService.saveDataToDbAndCache(cacheKey, count, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("agreed and disagreed: " + count + ", use cache: " + (cache != null));
        return count;
    }

    public long getClickCount(long beginTs, long endTs) throws Exception {
        String cacheKey = "click_count_" + beginTs + "_" + endTs;
        long click;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            click = (long) cache;
        } else {
            click = esService.getActionCount(Action.CLICK, beginTs, endTs);
            cacheService.saveDataToDbAndCache(cacheKey, click, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("click count: " + click + ", use cache: " + (cache != null));
        return click;
    }

    public List<DateCount> getClickCount(long beginTs, long endTs, String interval) throws Exception {
        String cacheKey = "click_count_" + interval + "_" + beginTs + "_" + endTs;
        List<DateCount> counts;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            counts = (List<DateCount>) cache;
        } else {
            counts = esService.getActionHistogram(Action.CLICK, null, beginTs, endTs, interval, null);
            cacheService.saveDataToDbAndCache(cacheKey, counts, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("click count: " + counts + ", use cache: " + (cache != null));
        return counts;
    }

    public List<ClickRatioCount> getClickRatio(long beginTs, long endTs, String interval) throws Exception {
        String cacheKey = "click_ratio_" + interval + "_" + beginTs + "_" + endTs;
        List<ClickRatioCount> resultList;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            resultList = (List<ClickRatioCount>) cache;
        } else {
            Map<String, ClickRatioCount> countsMap = new HashMap<>();
            String[] cardinalities = new String[]{FIELD_LOGID};
            List<DateCount> clickList = esService.getActionHistogram(Action.CLICK, cardinalities, beginTs, endTs,
                    interval, null);
            List<DateCount> totalList = esService.getActionHistogram(Action.SEARCH, cardinalities, beginTs, endTs,
                    interval, null);

            for (DateCount dateCount : clickList) {
                ClickRatioCount count = new ClickRatioCount();
                count.setDate(dateCount.getDate());
                count.setClick(dateCount.getCount());
                countsMap.put(dateCount.getDate(), count);
            }

            for (DateCount dateCount : totalList) {
                ClickRatioCount count = countsMap.get(dateCount.getDate());
                if (count == null) {
                    LOGGER.error("data err, date: {}", dateCount.getDate());
                } else {
                    long total = dateCount.getCount();
                    count.setTotal(total);
                    float ratio = (float) count.getClick() / total;
                    count.setRatio(ratio);
                }
            }

            resultList = new ArrayList<>(countsMap.values());
            resultList.sort(Comparator.comparing(ClickRatioCount::getDate));
            cacheService.saveDataToDbAndCache(cacheKey, resultList, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("click count: " + resultList + ", use cache: " + (cache != null));
        return resultList;
    }

    public long getSearchCount(long beginTs, long endTs) throws Exception {
        long pv;
        String cacheKey = "search_count_" + beginTs + "_" + endTs;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            pv = (long) cache;
        } else {
            pv = esService.getDistinctCount(Action.SEARCH, FIELD_LOGID, beginTs, endTs, null);
            cacheService.saveDataToDbAndCache(cacheKey, pv, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("pv: " + pv + ", use cache: " + (cache != null));
        return pv;
    }

    public long getHitQueryCount(long beginTs, long endTs) throws Exception {
        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(FIELD_SIZE, 0))
                .must(QueryBuilders.existsQuery(FIELD_QUERY)).must(QueryBuilders.existsQuery(FIELD_SIZE));
        String cacheKey = "hit_count_" + beginTs + "_" + endTs;
        long hit;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            hit = (long) cache;
        } else {
            hit = esService.getDistinctCount(Action.SEARCH, FIELD_LOGID, beginTs, endTs, queryBuilder);
            cacheService.saveDataToDbAndCache(cacheKey, hit, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("hit count: " + hit + ", use cache: " + (cache != null));
        return hit;
    }

    public Long getTurnQueryCount(long beginTs, long endTs) throws Exception {
        String cacheKey = "turn_count_" + beginTs + "_" + endTs;
        Long turn;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, Long.class);
        if (cache != null) {
            turn = (Long) cache;
        } else {
            BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(FIELD_PAGENUM, 1))
                    .must(QueryBuilders.existsQuery(FIELD_QUERY)).must(QueryBuilders.existsQuery(FIELD_PAGENUM));
            turn = esService.getDistinctCount(Action.SEARCH, FIELD_LOGID, beginTs, endTs, queryBuilder);
            cacheService.saveDataToDbAndCache(cacheKey, turn, LocalCache.EXPIRE_UNIT_DAY);
        }
        LOGGER.debug("hit count: " + turn + ", use cache: " + (cache != null));
        return turn;
    }

    public List<String> getHotQueries() throws Exception {
        List<QueryCount> queryCounts = getHotQueries(5);
        List<String> queryList = new ArrayList<>();
        for (QueryCount queryCount : queryCounts) {
            queryList.add(queryCount.getQuery());
        }
        return queryList;
    }

    public List<QueryCount> getHotQueries(int size) {
        String cacheValue = redisClient.getCache("hotData","hotquery");
        if (StringUtils.isEmpty(cacheValue)){
            LOGGER.info("cacheValue is null");
            return Collections.emptyList();
        }
        List<QueryCount> queryCounts = JacksonUtil.getObjectFromJson(cacheValue, new TypeReference<List<QueryCount>>() {
        });
        size = Math.min(size,queryCounts.size());
        return queryCounts.subList(0,size);
    }

    public List<QueryCount> getHotQueries(long beginTs, long endTs, int size) throws Exception {
        List<QueryCount> queries = new ArrayList<>();
        queries.clear();
        long totalCount = getPageView(beginTs, endTs);
        QueryBuilder queryBuilder = QueryBuilders.termQuery(FIELD_PAGENUM, 1);
        Range.Bucket rangeBucket =
                esService.getTopWords(Action.SEARCH, FIELD_QUERY, beginTs, endTs, size, queryBuilder);
        Aggregation termAgg = rangeBucket.getAggregations().get("count");
        if (termAgg == null) {
            throw new IOException("no data in elasticsearch");
        }
        for (Terms.Bucket termBucket : ((Terms) termAgg).getBuckets()) {
            Cardinality cardinality = termBucket.getAggregations().get("cardinality");
            QueryCount queryCount = new QueryCount(termBucket.getKeyAsString(), cardinality.getValue());
            float ratio = (float) cardinality.getValue() / totalCount;
            queryCount.setRatio(ratio);
            queries.add(queryCount);
        }
        queries.sort((o1, o2) -> (int) (o2.getCount() - o1.getCount()));
        LOGGER.info("hot queries: " + queries);
        size = Math.min(size, queries.size());
        return queries.subList(0, size);
    }

    // 定时计算热点数据
    public void cacheHotQueries(long beginTs, long endTs, int size) throws Exception {
        String cacheKey = "hotquery";
        List<QueryCount> queries = new ArrayList<>();
        long totalCount = getSearchCount(beginTs, endTs);
        QueryBuilder queryBuilder = QueryBuilders.termQuery(FIELD_PAGENUM, 1);
        Range.Bucket rangeBucket = esService.getTopWords(Action.SEARCH, FIELD_QUERY, beginTs, endTs, size,
                queryBuilder);
        Aggregation termAgg = rangeBucket.getAggregations().get("count");
        if (termAgg == null) {
            throw new IOException("no data in elasticsearch");
        }
        for (Terms.Bucket termBucket : ((Terms) termAgg).getBuckets()) {
            Cardinality cardinality = termBucket.getAggregations().get("cardinality");
            QueryCount queryCount = new QueryCount(termBucket.getKeyAsString(), cardinality.getValue());
            float ratio = (float) cardinality.getValue() / totalCount;
            queryCount.setRatio(ratio);
            queries.add(queryCount);
        }
        queries.sort((o1, o2) -> (int) (o2.getCount() - o1.getCount()));
        LOGGER.info("hot queries: " + queries);
        redisClient.setCache("hotData", cacheKey, JacksonUtil.toJson(queries), 1, TimeUnit.DAYS);
    }

    public List<QueryCount> getHotMissQueries(long beginTs, long endTs, int size) throws Exception {
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(FIELD_SIZE, 0);

        String cacheKey = "missquery_" + beginTs + "_" + endTs;
        boolean useCache = false;
        List<QueryCount> queries = new ArrayList<>();
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            queries = (List<QueryCount>) cache;
            if (queries.size() >= size) {
                useCache = true;
            }
        }

        if (!useCache) {
            queries.clear();
            Range.Bucket rangeBucket = esService.getTopWords(Action.SEARCH, FIELD_QUERY, beginTs, endTs, size,
                    termQueryBuilder);
            Aggregation termAgg = rangeBucket.getAggregations().get("count");
            if (termAgg == null) {
                throw new IOException("no data in elasticsearch");
            }
            for (Terms.Bucket termBucket : ((Terms) termAgg).getBuckets()) {
                Cardinality cardinality = termBucket.getAggregations().get("cardinality");
                queries.add(new QueryCount(termBucket.getKeyAsString(), cardinality.getValue()));
            }
            queries.sort((o1, o2) -> (int) (o2.getCount() - o1.getCount()));
            cacheService.saveDataToDbAndCache(cacheKey, queries, LocalCache.EXPIRE_UNIT_HOUR);
        }

        LOGGER.debug("hot queries: " + queries + ", use cache: " + useCache);
        size = Math.min(size, queries.size());
        return queries.subList(0, size);

    }

    public List<QueryCount> getNoClickQueries(long beginTs, long endTs, int size) throws Exception {
        List<QueryCount> resultList = new ArrayList<>();

        String cacheKey = "noclickqurey_" + beginTs + "_" + endTs;
        boolean useCache = false;
        Object cache = cacheService.getDataFromDbOrCache(cacheKey, ArrayList.class);
        if (cache != null) {
            resultList = (List<QueryCount>) cache;
            if (resultList.size() >= size) {
                useCache = true;
            }
        }
        if (!useCache) {
            resultList.clear();
            Action[] actions = new Action[]{Action.SEARCH, Action.CLICK};
            // 使用脚本对query和logId进行联合去重
            String script = "if(doc['" + FIELD_LOGID + "'].size()  == 0)  return ''; " + "if(doc['" + FIELD_QUERY
                    + "'].size() == 0) return '[error]_'+doc['" + FIELD_LOGID + "'].value; " + "return doc['"
                    + FIELD_QUERY + "'].value+'_'+doc['" + FIELD_LOGID + "'].value";
            String cardCondition = "params._value0 < 2";
            // todo: 数据返回后，还需要进行二次处理，算法待优化
            Range.Bucket bucketResult = esService.getMultiTopWords(actions, script, cardCondition, beginTs, endTs,
                    Integer.MAX_VALUE - 1);
            Aggregation termAgg = bucketResult.getAggregations().get("terms");
            if (termAgg == null) {
                throw new IOException("no data in elasticsearch");
            }
            Map<String, Integer> queryMap = new HashMap<>();
            for (Terms.Bucket termBucket : ((Terms) termAgg).getBuckets()) {
                Cardinality cardinality = termBucket.getAggregations().get("cardinality");
                // 小于2的为没有点击的query
                if (cardinality.getValue() < 2) {
                    String[] keyArr = termBucket.getKeyAsString().split("_");
                    String query = keyArr[0].replace("[", "").replace("]", "");
                    if (query.equals("error")) {
                        continue;
                    }
                    if (queryMap.containsKey(query)) {
                        queryMap.put(query, queryMap.get(query) + 1);
                    } else {
                        queryMap.put(query, 1);
                    }
                } else {
                    LOGGER.info("aggregate data may be error, query_logId: {}, cardinality num: {}, ",
                            termBucket.getKeyAsString(), cardinality.getValue());
                }
            }
            LOGGER.debug("no click result map:{}", queryMap);

            List<QueryCount> finalResultList = resultList;
            queryMap.forEach((key, value) -> finalResultList.add(new QueryCount(key, value)));

            resultList.sort((o1, o2) -> (int) (o2.getCount() - o1.getCount()));
            cacheService.saveDataToDbAndCache(cacheKey, resultList, LocalCache.EXPIRE_UNIT_HOUR);
        }
        size = Math.min(size, resultList.size());
        LOGGER.debug("search list size: " + resultList.size() + ", use cache: " + useCache);
        return resultList.subList(0, size);
    }

    public List<KnowledgeMapDTO> cacheNodeCounts() {
        log.info("===statistics cacheNodeCounts start===");
        List<TreeNode> treeNodes = treeService.getTreeNodes();
        List<KnowledgeMapDTO> resultList = new ArrayList<>();
        treeNodes.forEach(treeNode -> {
            if (1 == treeNode.getId()) {
                return;
            }
            KnowledgeMapDTO knowledgeMapDTO = new KnowledgeMapDTO();
            knowledgeMapDTO.setSystemId(treeNode.getId());
            knowledgeMapDTO.setSystemName(treeNode.getName());
            int total = 0;
            SourceService.SearchCriteria sc = createCommonSearchCriteria(treeNode);
            total = executeKGMapTotal(total, sc, knowledgeMapDTO);
            knowledgeMapDTO.setDataCount(total);
            resultList.add(knowledgeMapDTO);
        });
        redisClient.setCache(CACHEKG, CACHEKGMAPCOUNT, JacksonUtil.toJson(resultList), 1, TimeUnit.DAYS);
        log.info("===statistics cacheNodeCounts end===:{}", resultList);
        return resultList;
    }

    public void syncShenFeiUserInfo(String req) {
        log.info("===syncShenFeiUserInfo start===");
        // 全量更新avic_user
        try {
            if (StringUtils.isEmpty(req)) {
                req = "<ReqMsg>" +
                        "<document>" +
                        "<![CDATA[" +
                        "<info>" +
                        "<appCodeName>appPORTALLoginEnable</appCodeName>" +
                        "<startDn>ou=orgs,ou=131601,o=avic</startDn>" +
                        "<scope>2</scope>" +
                        "</info>]]>" +
                        "</document>" +
                        "</ReqMsg>";
            }
            log.info("===syncShenFeiUserInfo req===:{}", req);
            log.info("===syncShenFeiUserInfo ldapUrl===:{}", ldapUrl);
            log.info("===syncShenFeiUserInfo personMethod===:{}", personMethod);
            log.info("===syncShenFeiUserInfo nameSpace===:{}", nameSpace);
            String resultStr = SOAPClient.splicingMessage(ldapUrl
                    , personMethod
                    , nameSpace
                    , req);
            log.info("===remote syncShenFeiUserInfo response===:{}", resultStr);
            if (StringUtils.isEmpty(resultStr)) {
                log.error("===syncShenFeiUserInfo ERROR===:{}", "请求结果为空");
            }
            JSONObject objectList = XML.toJSONObject(resultStr);
            Object respMsgObj = objectList.get("respMsg");
            Object documentobj = ((JSONObject) respMsgObj).get("document");
            JSONObject documentJsonObject = XML.toJSONObject(documentobj.toString());
            Object usersObj = documentJsonObject.get("users");
            JSONArray userJsonArray = (JSONArray) ((JSONObject) usersObj).get("user");
            if (ObjectUtils.isEmpty(userJsonArray)) {
                return;
            } else {
                enumDao.clearAvicUserInfo();
            }
            this.saveAvicUser(userJsonArray);
            log.info("===syncShenFeiUserInfo end===");
        } catch (Exception e) {
            log.error("===syncShenFeiUserInfo ERROR===", e);
        }

    }

    private void saveAvicUser(JSONArray userJsonArray) {
        for (Object jsonObject : userJsonArray) {
            JSONObject temp = (JSONObject) jsonObject;
            AvicUserDTO avicUserDTO = new AvicUserDTO();
            avicUserDTO.setId(UUID.randomUUID().toString());
            avicUserDTO.setAvicDeptCode(!temp.has("avicDeptCode") ? null
                    : temp.get("avicDeptCode").toString());
            avicUserDTO.setAvicDeptName(!temp.has("avicDeptName") ? null
                    : temp.get("avicDeptName").toString());
            if (temp.has("avicUserSecurity")) {
                String avicUserSecurity = temp.get("avicUserSecurity").toString();
                convertUserSecurity(avicUserDTO, avicUserSecurity);
            }
            if (!temp.has("avicUserAccount")) {
                continue;
            }
            avicUserDTO.setAvicUserAccount(ObjectUtils.isEmpty(temp.get("avicUserAccount")) ? null
                    : temp.get("avicUserAccount").toString());
            avicUserDTO.setFullname(!temp.has("fullName") ? null
                    : temp.get("fullName").toString());
            avicUserDTO.setMail(!temp.has("mail")
                    ? temp.get("avicUserAccount").toString() + "@sy601.com" : temp.get("mail").toString());
            enumDao.insertAvicUserInfo(avicUserDTO);
        }
    }

    public void convertUserSecurity(AvicUserDTO avicUserDTO, String avicUserSecurity) {
        switch (avicUserSecurity) {
            case "0":
                avicUserDTO.setAvicUserSecurity("非涉密人员");
                break;
            case "1":
                avicUserDTO.setAvicUserSecurity("一般涉密人员");
                break;
            case "2":
                avicUserDTO.setAvicUserSecurity("重要涉密人员");
                break;
            case "3":
                avicUserDTO.setAvicUserSecurity("核心涉密人员");
                break;
            default:
                break;
        }
    }

    public void syncShenFeiOrgInfo(String req) {
        log.info("===syncShenFeiOrgInfo start===");
        // 全量更新avic_dept表
        try {
            if (StringUtils.isEmpty(req)) {
                req = "<ReqMsg>" +
                        "<document>" +
                        "<![CDATA[" +
                        "<info>" +
                        "<appCodeName>appPORTALLoginEnable</appCodeName>" +
                        "<startDn>ou=orgs,ou=131601,o=avic</startDn>" +
                        "<scope>2</scope>" +
                        "</info>]]>" +
                        "</document>" +
                        "</ReqMsg>";
            }
            log.info("===syncShenFeiOrgInfo req===:{}", req);
            log.info("===syncShenFeiOrgInfo ldapUrl===:{}", ldapUrl);
            log.info("===syncShenFeiOrgInfo personMethod===:{}", personMethod);
            log.info("===syncShenFeiOrgInfo nameSpace===:{}", nameSpace);
            String resultStr = SOAPClient.splicingMessage(ldapUrl
                    , deptMethod
                    , nameSpace
                    , req);
            log.info("===remote syncShenFeiOrgInfo response===:{}", resultStr);
            if (StringUtils.isEmpty(resultStr)) {
                log.error("===syncShenFeiOrgInfo ERROR===:{}", "请求结果为空");
            }
            JSONObject objectList = XML.toJSONObject(resultStr);
            Object respMsgObj = objectList.get("respMsg");
            Object documentobj = ((JSONObject) respMsgObj).get("document");
            JSONObject documentJsonObject = XML.toJSONObject(documentobj.toString());
            Object deptsObj = documentJsonObject.get("depts");
            JSONArray deptJsonArray = (JSONArray) ((JSONObject) deptsObj).get("dept");
            if (ObjectUtils.isEmpty(deptJsonArray)) {
                return;
            } else {
                enumDao.clearAvicOrgInfo();
            }
            this.saveAvicDept(deptJsonArray);
            log.info("===syncShenFeiOrgInfo end===");
        } catch (Exception e) {
            log.error("===syncShenFeiOrgInfo ERROR===", e);
        }

    }

    public static boolean isInteger(String str) {
        Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$");
        return pattern.matcher(str).matches();
    }

    private void saveAvicDept(JSONArray deptJsonArray) {
        for (Object jsonObject : deptJsonArray) {
            JSONObject temp = (JSONObject) jsonObject;
            AvicOrgDTO avicOrgDTO = new AvicOrgDTO();
            avicOrgDTO.setId(UUID.randomUUID().toString());
            if (temp.has("avicDeptCode") && !isInteger(temp.get("avicDeptCode").toString())) {
                continue;
            }
            if (temp.has("avicParentDeptCode") && !isInteger(temp.get("avicParentDeptCode").toString())) {
                continue;
            }
            avicOrgDTO.setAvicDeptCode(!temp.has("avicDeptCode") ? null
                    : temp.get("avicDeptCode").toString());
            avicOrgDTO.setAvicDeptName(!temp.has("avicDeptName") ? null
                    : temp.get("avicDeptName").toString());
            avicOrgDTO.setAvicParentDeptCode(!temp.has("avicParentDeptCode") ? null
                    : temp.get("avicParentDeptCode").toString());
            avicOrgDTO.setAvicParentDeptName(!temp.has("avicParentDeptName") ? null
                    : temp.get("avicParentDeptName").toString());
            avicOrgDTO.setAvicDeptState(!temp.has("avicDeptState") ? null
                    : temp.get("avicDeptState").toString());
            enumDao.insertAvicOrgInfo(avicOrgDTO);
        }
    }

    private int executeKGMapTotal(int total, SourceService.SearchCriteria sc, KnowledgeMapDTO knowledgeMapDTO) {
        sc.setContentType(SourceContentType.FILE.getName());
        Map<String, Object> file = sourceService.getListFromMySql(sc, "statistic");
        total = getDataSize(total, knowledgeMapDTO, file);
        sc.setContentType(SourceContentType.STRUCT.getName());
        Map<String, Object> struct = sourceService.getStructList(sc);
        if (!ObjectUtils.isEmpty(struct) && !ObjectUtils.isEmpty(struct.get("totalSize"))) {
            total += Integer.parseInt(struct.get("totalSize").toString());
        }
        sc.setContentType(SourceContentType.VIDEO.getName());
        Map<String, Object> video = sourceService.getListFromMongo(sc);
        if (!ObjectUtils.isEmpty(video) && !ObjectUtils.isEmpty(video.get("totalSize"))) {
            total += Long.parseLong(video.get("totalSize").toString());
        }
        sc.setContentType(SourceContentType.WEB.getName());
        Map<String, Object> web = sourceService.getListFromMongo(sc);
        if (!ObjectUtils.isEmpty(web) && !ObjectUtils.isEmpty(web.get("totalSize"))) {
            total += Long.parseLong(web.get("totalSize").toString());
        }
        return total;
    }

    public int getDataSize(int total, KnowledgeMapDTO knowledgeMapDTO, Map<String, Object> file) {
        Long dataSize = 0L;
        if (!ObjectUtils.isEmpty(file) && !ObjectUtils.isEmpty(file.get("totalSize"))) {
            total += Long.parseLong(file.get("totalSize").toString());
            List<FileDataListDto> list = (List<FileDataListDto>) file.get("list");
            for (FileDataListDto fileDataListDto : list) {
                dataSize += fileDataListDto.getSize();
                if (null != fileDataListDto.getAttachments() && fileDataListDto.getAttachments().size() > 0) {
                    for (FileAttachment attachment : fileDataListDto.getAttachments()) {
                        dataSize += attachment.getSize();
                    }
                }
            }
        }
        knowledgeMapDTO.setDataSize(dataSize);
        return total;
    }

    @NotNull
    private SourceService.SearchCriteria createCommonSearchCriteria(TreeNode treeNode) {
        SourceService.SearchCriteria sc = new SourceService.SearchCriteria();
        sc.setNodeId(treeNode.getId());
        sc.setInputTypes(Arrays.asList("upload", "api_push", "api_pull"));
        sc.setPageNumber(1);
        sc.setPageSize(1);
        return sc;
    }

    public List<KnowledgeMapDTO> getKgMqpData() {
        List<KnowledgeMapDTO> result = new ArrayList<>();
        String cacheValue = redisClient.getCache(CACHEKG, CACHEKGMAPCOUNT);
        if (StringUtils.isEmpty(cacheValue)) {
            LOGGER.info("getKgMqpData cache is null");
            return Collections.emptyList();
        }
        result = JacksonUtil.getObjectFromJson(cacheValue, new TypeReference<List<KnowledgeMapDTO>>() {
        });
        return result;
    }

    public List<String> dataTimelinessCheck() {
        // 查询新闻同步时间
        List<String> alertTask = new ArrayList<>();
        List<ShenFeiDataDto> shenFeiNews = enumDao.getShenFeiNews();
        checkTimeliness(shenFeiNews, alertTask, "新闻");
        // 查询新闻上次接入时间
        if (alertTask.size() == 0) {
            String[] systems = needCheckSystem.split(",");
            for (String system : systems) {
                DataAccessSource dataAccessSourceInfo =
                        dataAccessSourceDao.findDataAccessSourceById(Integer.valueOf(system));
                if (dataAccessSourceInfo != null) {
                    if (
                            DateUtil.differentDaysByMillisecond(
                                    new Date(dataAccessSourceInfo.getTriggerLastTime()), new Date()) >= 3) {
                        alertTask.add(dataAccessSourceInfo.getName());
                    }
                }
            }
        }
        // 是否需要发送邮件，时间比较
        return alertTask;
    }

    private void checkTimeliness(List<ShenFeiDataDto> shenFeiDataDtos, List<String> aleryTask, String type) {
        if (shenFeiDataDtos.size() > 0) {
            if (!shenFeiDataDtos.get(0).checkTimeliness()) {
                aleryTask.add(type);
            }
        }
    }

}
